package storage.v1;

import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;

import com.onpositive.data.IPropertyConstants;
import com.onpositive.data.units.DateWithAccuracy;
import com.onpositive.data.units.Unit;
import com.onpositive.knowledge.model.IThing;
import com.onpositive.knowledge.model.IType;
import com.onpositive.mediawiki.propertyparser.ParsedCategory;
import com.onpositive.mediawiki.propertyparser.ParsedDocument;
import com.onpositive.mediawiki.propertyparser.PropertyCharacteristic;
import com.onpositive.semantic.model.api.meta.IMeta;
import com.onpositive.semantic.model.api.property.IProperty;
import com.onpositive.semantic.model.api.property.IPropertyProvider;
import com.onpositive.semantic.model.api.realm.IRealm;
import com.onpositive.units.AbstractPreprocessor;
import com.onpositive.units.AbstractValueParser;
import com.onpositive.units.ParsedDate;
import com.onpositive.units.ParsedDimension;
import com.onpositive.units.ParsedProperty;
import com.onpositive.units.ParsedScalar;
import com.onpositive.units.ParsedString;
import com.onpositive.units.ParsedValue;
import com.onpositive.units.ParsedValueSequence;
import com.onpositive.units.RangeValue;

public class BasicTableInfoProcessor {
	
	private static final String TARGET_NAME_DOCS = "docNames";
	private static final String TARGET_NAME_PROPS = "propNames";


	public BasicTableInfoProcessor(CategoryStorage storage) {
		super();
		this.storage = storage;
		ParsedCategory category = storage.getCategory();
		if(category!=null){
			Collection<String> docNames  = category.getDocumentNamesList() ;
			Collection<String> propNames = category.getPropertyNames() ;
			
			docsMatcher = new Matcher( docNames ){
				@Override
				protected String canonize( String s ){
					String str = super.canonize(s) ;
					String redirected = ParserV2.redirect(str) ;
					if(redirected == null)
						redirected = str ;
					return redirected ;
				}
			} ;
			propsMatcher = new Matcher( propNames ) ;
		}
		else{
			docsMatcher = new Matcher(new ArrayList<String>());
			propsMatcher = new Matcher(new ArrayList<String>());
		}
	}

	final Matcher docsMatcher ;
	final Matcher propsMatcher ;
	final CategoryStorage storage ;
	private HashMap<String,Integer> propNamesDeviations = new HashMap<String, Integer>() ;
	
	
	public void processTableInfo()
	{		
		ArrayList<ITableInfo> tableInfoCollection = storage.getTableInfoList() ;
		
		for( ITableInfo ti : tableInfoCollection )
		{
			propNamesDeviations.clear() ;
			if( !(ti instanceof BasicTableInfo) )
				continue;
			
			BasicTableInfo bti = (BasicTableInfo) ti ;
			try{			
				processSingleInfo(bti) ;
			}catch(Exception e){				
				e.printStackTrace() ;
				System.out.println( bti.getDocumentName() +" has table Error." ) ;
			}
		}								
	}	
	

	private void processSingleInfo(BasicTableInfo bti)
	{
		final String tableDocName = bti.getDocumentName();
		System.out.println( tableDocName );
		
		OrientationInfo oi = normalize( bti ) ;		
		if( oi == null )
			return ;
		
		ArrayList<ArrayList<String>> normalizedContent = oi.normalized ;
		if( normalizedContent == null
				|| normalizedContent.size() < 2
				|| normalizedContent.get(0) == null 
				|| normalizedContent.get(1) == null )
			return ;
		
		if( oi.targetName == TARGET_NAME_PROPS ){
			
			ParsedDocument pDoc = storage.getCategory().getParsedDocument( bti.getDocumentName() ) ;
			if(pDoc == null)
				return ;
			
			appendContent( pDoc, normalizedContent ) ;			
			return ;
		}		
		
		//content.get(0) is properties name list
		//content.get(1) is object's names list
		//content.get(i) is object's values list for content.get(0).get(i-2) property 
		
		ParsedCategory temporaryCategory = buildCategory( normalizedContent, tableDocName ) ;
		merge( temporaryCategory, normalizedContent, tableDocName ) ;
	}


	private void appendContent( ParsedDocument pDoc, ArrayList<ArrayList<String>> normalizedContent) {
		
		IParserIdLookup pil = createparserIdLookUp(storage) ;
		IParserProvider dpp = DefaultParserProvider.getInstance() ;
		StringBuilder bld = new StringBuilder() ;
		String comment = composeComment( pDoc.getName(), 2 ) ;
		
		for( ArrayList<String> lst : normalizedContent ){
			
			String propName = lst.get(0).toLowerCase() ;
			PropertyCharacteristic pChar = storage.getCategory().getCharacteristics(propName) ;
			if( pChar == null )
				continue ;
			
			String propValue = lst.get(1) ;
			if( propValue == null || propValue.length() == 0 )
				continue ;
			
			bld.append( propName ) ;
			bld.append( " = " ) ;
			bld.append( propValue ) ;
			
			String inputString = bld.toString() ;
			bld.delete(0, bld.length() ) ;				
							
			ParsedProperty newProp = new ValuesExtractorV1().processProperty(
					dpp,
					inputString,
					null,
					pil,
					this.storage,
					comment ) ;
			
			ParsedProperty oldProp = pDoc.getProperty(propName) ;
			if( oldProp == null )
				pDoc.addProperty(newProp) ;
			else{
				for( ParsedValue pv : newProp.getValues() )
					oldProp.addValue(pv) ;
			}
		}
				
	}

	private HashMap<String,ArrayList<ParsedValue>> temporaryValueStorage = 
			new HashMap<String, ArrayList<ParsedValue>>() ; 

	private void merge(
			ParsedCategory temporaryCategory,
			ArrayList<ArrayList<String>> normalizedContent,
			String tableDocName)
	{
		final ParsedCategory oldCategory = storage.getCategory();
		ArrayList<String> propNames = normalizedContent.get(0) ;
		ArrayList<String> docNames = normalizedContent.get(1) ;
		
		for( Map.Entry<String, PropertyCharacteristic> entry0 :
							   temporaryCategory.getChracteristicsMap().entrySet() )
		{
			temporaryValueStorage.clear() ;
			String propName = entry0.getKey() ;
			PropertyCharacteristic newChar = entry0.getValue() ;
			
			int rate = -1 ;
			PropertyCharacteristic oldChar = oldCategory.getCharacteristics(propName) ;
			
			if( oldChar != null ){
				int newRate = compare( newChar, oldChar, temporaryCategory, normalizedContent, tableDocName ) ;
				rate = newRate > rate ? newRate : rate ;
				
				if( rate >= 0 ){
					mergeValueArrays( oldChar, docNames ) ;
					continue ;
				}
			}
				
			final Integer devInd = propNamesDeviations.get(propName);
			String oldName = propNames.get( devInd );
			
			oldChar = oldCategory.getCharacteristics( oldName ) ;
			if( oldChar != null ){
				int newRate = compare( newChar, oldChar, temporaryCategory, normalizedContent, tableDocName ) ;
				rate = newRate > rate ? newRate : rate ;
				if( rate >= 0 ){
					mergeValueArrays( oldChar, docNames ) ;
					continue ;
				}
			}		
			
			PropertyCharacteristic bestOldChar = null ;
			for( Map.Entry<String, PropertyCharacteristic> entry1 : oldCategory.getChracteristicsMap().entrySet() )
			{
				oldChar = entry1.getValue() ;
				int newRate = compare( newChar, oldChar, temporaryCategory, normalizedContent, tableDocName ) ;
				if( newRate >= 0 ){
					rate = newRate > rate ? newRate : rate ;
					bestOldChar = oldChar ;
				}
			}
			if( rate >= 0 )
				mergeValueArrays( bestOldChar, docNames ) ;
		}		
	}


	private void mergeValueArrays(PropertyCharacteristic oldChar,ArrayList<String> docNames)
	{
		String oldPropName = oldChar.getName() ;
		String oldParserId = oldChar.getCharacteristic( IPropertyConstants.CHAR_PARSER_ID_LABEL ) ;
		ArrayList<ParsedValue> newVals = temporaryValueStorage.get(oldParserId) ;

		final int docsCount = docNames.size();
		for( int i = 0 ; i < docsCount ; i++ )
		{
			ParsedValue newValue = newVals.get(i) ;
			String docName = docNames.get(i) ;			
			ParsedDocument pDoc = this.storage.getParsedDocument(docName) ;
			if( pDoc == null )
				continue ;

			ParsedProperty pProp = pDoc.getProperty(oldPropName) ;
			if( pProp == null ){
				pProp = new ParsedProperty(oldPropName) ;
				pDoc.addProperty(pProp) ;
			}
			pProp.addValue( newValue ) ;
		}
	}

	
	private HashMap<String,Double> epsilonMap = new HashMap<String, Double>() ;
	
	private int compare(PropertyCharacteristic newChar,
						PropertyCharacteristic oldChar,
						ParsedCategory temporaryCategory,
						ArrayList<ArrayList<String>> normalizedContent,
						String tableDocName )
	{
		String oldParserId = oldChar.getCharacteristic( IPropertyConstants.CHAR_PARSER_ID_LABEL ) ;
		String newParserId = newChar.getCharacteristic( IPropertyConstants.CHAR_PARSER_ID_LABEL ) ;
		
		if( !compatible( oldParserId, newParserId ) )
			return -1 ;
		
		String propName = oldChar.getName() ;
		
		String comment = composeComment(tableDocName, 1) ;
		
		ArrayList<ParsedValue> valList = prepareValuesList(newChar, oldChar, temporaryCategory, normalizedContent, comment );
		ArrayList<String> docNames = normalizedContent.get(1) ;
		
		int docsCount = docNames.size();
		int determinedValuesCount = 0 ;
		int coincidenceCount = 0 ;
		
		Double epsilon = 0.0 ;
		if( newParserId.equals(ParsedScalar.ID) || newParserId.equals(ParsedDimension.ID) ){
			epsilon = epsilonMap.get(propName) ;
			if( epsilon == null ){
				epsilon = computeEpsilon( propName ) ;
				epsilonMap.put(propName, epsilon) ;
			}
			if( epsilon == null )
				return -1 ;
		}
		
		for( int i = 0 ; i < docsCount ; i++ )
		{
			ParsedValue newValue = valList.get(i) ;
			if( newValue == null )
				continue ;
			
			String docName = docNames.get(i) ;
			ParsedDocument pDoc = this.storage.getParsedDocument(docName) ;
			if( pDoc == null )
				continue ;

			ParsedProperty pProp = pDoc.getProperty(propName) ;
			if( pProp == null )
				continue ;

			final LinkedHashSet<ParsedValue> values = pProp.getValues();
			if( values == null || values.isEmpty() )
				continue ;

			determinedValuesCount++ ;
			for( ParsedValue pv : values )
			{
				if( !(pv.getId().equals(newValue.getId())) )
					continue ;
				
				if( areEqual( pv, newValue, epsilon ) )
					coincidenceCount++ ;
			}			
		}
		
		if( coincidenceCount == 0 )
			return -1 ;
		
		int persentage = (int)(100*((double)coincidenceCount)/determinedValuesCount + 0.5 ) ;		
		return persentage ;
	}

	@SuppressWarnings("unchecked")
	private boolean areEqual( ParsedValue pv, ParsedValue newValue, double epsilon ) {
		
		String id = pv.getId() ;
		if( id.equals( ParsedScalar.ID) )
		{
			ParsedScalar oldPS = (ParsedScalar) pv ;
			ParsedScalar newPS = (ParsedScalar) newValue ;
			
			Object oldDoubleValue = oldPS.value() ;
			Object newDoubleValue = newPS.value() ;
			
			return compareDouble( oldDoubleValue, newDoubleValue , epsilon ) ;			
		}
		else if( id.equals( ParsedDimension.ID) )
		{
			ParsedDimension oldPD = (ParsedDimension) pv ;
			ParsedDimension newPD = (ParsedDimension) newValue ;
			
			Unit oldUnit = oldPD.getUnit() ;
			Unit newUnit = newPD.getUnit() ;
			
			if( !(oldUnit.getKind().equals(newUnit.getKind())) )
				return false ;
			
			Object oldDoubleValue =  oldPD.value() ;
			Object newDoubleValue =  newPD.value() ;
			
			return compareDouble( oldDoubleValue, newDoubleValue , epsilon ) ;						
		}
		else if ( id.equals(ParsedDate.ID)){
			
			DateWithAccuracy oldDate = (DateWithAccuracy) pv.value() ;
			DateWithAccuracy newDate = (DateWithAccuracy) newValue.value() ;
			
			int oldYear = oldDate.getCalendar().get( GregorianCalendar.YEAR ) ;			
			int newYear = newDate.getCalendar().get( GregorianCalendar.YEAR ) ;
			return oldYear == newYear ;
		}
		else if ( id.equals(ParsedValueSequence.ID)){
			LinkedHashSet<ParsedString> set = (LinkedHashSet<ParsedString>)pv.value() ;
			ParsedString nVal = ((LinkedHashSet<ParsedString>)newValue.value()).iterator().next() ;
			for( ParsedString ps : set )
				if( areEqual( ps, nVal, 0.0 ) )
					return true ;			
		} else{
			final String oldValstring = pv.getValueAsString().toLowerCase().trim();
			final String newValString = newValue.getValueAsString().toLowerCase().trim();
			return oldValstring.equals( newValString ) ;			
		}

		return false;
	}


	private boolean compareDouble( Object oldVal, Object newVal,	double epsilon)
	{
		if( oldVal == null || newVal == null )
			return false ;
		
		Double oldDoubleVal = (Double) oldVal ;
		Double newDoubleVal = (Double) newVal ;
		
		return Math.abs( oldDoubleVal - newDoubleVal) < epsilon ;
	}


	private Double computeEpsilon( String propName )
	{
		double max = Double.MIN_VALUE ;
		double min = Double.MAX_VALUE ;
		Collection<ParsedDocument> allDocs = this.storage.getCategory().getDocumentsList() ;
		boolean gotValues = false ;
		double lb= 0.0, ub=0.0;
		boolean gotLb, gotUb ;
		for( ParsedDocument pDoc : allDocs )
		{
			ParsedProperty pProp = pDoc.getProperty(propName) ;
			if( pProp == null )
				continue ;
			
			LinkedHashSet<ParsedValue> valSet = pProp.getValues() ;
			if( valSet.isEmpty() )
				continue ;
			
			
			for( ParsedValue pv : valSet )
			{
				if( !( pv.getId().equals(ParsedScalar.ID) || pv.getId().equals(ParsedDimension.ID)) )
					continue ;

				gotLb = false ;
				gotUb = false ;
				
				if( pv instanceof RangeValue ){
					// TODO check isRange() here
					RangeValue rv = (RangeValue) pv;
					Object lbObj = rv.getLowerBound();
					if( lbObj != null )
						try{
							lb = (Double) lbObj ;
							gotLb = true ;
						} catch(ClassCastException e ){
							e.printStackTrace() ;
						}
					
					Object ubObj = rv.getUpperBound();
					if( ubObj != null )
						try{
							ub = (Double) ubObj ;
							gotUb = true ;
						} catch(ClassCastException e ){
							e.printStackTrace() ;
						}
				}
				else{
					Object valObj = pv.value() ;
					if(valObj != null)
						try{
							double dVal = (Double) valObj ;
							ub = dVal ;
							lb = dVal ;
							gotUb = true ;
							gotLb = true ;
						} catch(ClassCastException e ){
							e.printStackTrace() ;
						}					
				}
				if( !(gotLb||gotUb) )
					continue ;				
				
				
				if( gotUb && ub > max )
					max = ub ;
					
				if( gotLb && lb < min )
					min = lb ;
				
			}
		}		
		if( !gotValues )
			return null ;
		
		double epsilon = 0.001 * ( max - min ) ;
		return epsilon ;
	}


	private ArrayList<ParsedValue> prepareValuesList(
			PropertyCharacteristic newChar,
			PropertyCharacteristic oldChar,
			ParsedCategory temporaryCategory,
			ArrayList<ArrayList<String>> normalizedContent,
			String comment )
	{		
		final String oldParserId = oldChar.getCharacteristic( IPropertyConstants.CHAR_PARSER_ID_LABEL ) ;
		final String newParserId = newChar.getCharacteristic( IPropertyConstants.CHAR_PARSER_ID_LABEL ) ;
		
		ArrayList<ParsedValue> valList = this.temporaryValueStorage.get(oldParserId) ;
		if( valList == null )
		{
			String propName = newChar.getName() ;
			ArrayList<String> docNames = normalizedContent.get(1) ;
			valList = new ArrayList<ParsedValue>() ;
			if( oldParserId.equals( newParserId ) )
			{
				for( String s : docNames )
				{
					ParsedDocument pDoc = temporaryCategory.getParsedDocument(s) ;
					if( pDoc == null ){
						valList.add(null) ;
						continue ;
					}
					ParsedProperty pProp = pDoc.getProperty(propName) ;
					if( pProp == null ){
						valList.add(null) ;
						continue ;
					}
					final LinkedHashSet<ParsedValue> values = pProp.getValues();
					if( values == null || values.isEmpty() ){
						valList.add(null) ;
						continue ;
					}
					ParsedValue pVal = values.iterator().next() ;
					valList.add(pVal) ;					
				}				
			}else{
				ValuesExtractorV1 ve = new ValuesExtractorV1() ;
				IParserProvider pp = new IParserProvider() {
					
					@Override
					public AbstractValueParser getParser(String parserid) {
						return DefaultParserProvider.getInstance().getParser(oldParserId);
					}
				};
				
				final String val = newChar.getCharacteristic("indexInNamesArray" );
				int ind = Integer.parseInt( val ) ;
				ArrayList<String> valueStringArray = normalizedContent.get(ind+2) ;
				String unprocesedPropertyName = normalizedContent.get(0).get(ind) ;
				StringBuilder bld = new StringBuilder() ;
				bld.append(unprocesedPropertyName) ;
				bld.append(" = ") ;
				int valStart = bld.length() ;
				
				if( docNames.size() == valueStringArray.size() )
				{				
					for( int i = 0 ; i < docNames.size() ; i++ )
					{
						String valString = valueStringArray.get(i) ;
						if( valString == null || valString.length() == 0 ){
							valList.add(null) ;
							continue ;
						}
						bld.delete(valStart, bld.length() ) ;
						bld.append( valString ) ;
						final String inputString = bld.toString();
						
						ParsedProperty pProp = ve.processProperty(
								pp,
								inputString,
								null,
								this.storage,
								this.storage,
								comment  ) ;
						
						if( pProp == null ){
							valList.add(null) ;
							continue ;
						}
						final LinkedHashSet<ParsedValue> values = pProp.getValues();
						if( values == null || values.isEmpty() ){
							valList.add(null) ;
							continue ;
						}
						ParsedValue pVal = values.iterator().next() ;
						valList.add(pVal) ;
					}
				}
			}
			temporaryValueStorage.put(oldParserId, valList) ;
		}
		return valList ;
	}

	private HashSet<String> compatibilitySet = new HashSet<String>(
		Arrays.asList( new String[]{
				// old + new
				ParsedDimension.ID + ParsedScalar.ID,
				ParsedDate.ID      + ParsedScalar.ID,
		})	) ;
	
	private HashSet<String> goodSet = new HashSet<String>(
			Arrays.asList( new String[]{
					// old + new
					ParsedString.ID,
					ParsedValueSequence.ID
			})	) ;

	private boolean compatible(String oldParserId, String newParserId)
	{
		if( newParserId.equals(oldParserId) )
			return true ;
		
		if( goodSet.contains( oldParserId ) )
			return true ;
		
		if( compatibilitySet.contains(oldParserId + newParserId) )
			return true ;
		
		return false;
	}


	private ParsedCategory buildCategory( ArrayList<ArrayList<String>> normalizedContent, String tableDocName )
	{		
		propNamesDeviations.clear() ;
		CategoryStorage cs = createEmptyCategoryStorage();
		
		final ProperyDataExtractor pe = new ProperyDataExtractor();
		
		String comment = composeComment( tableDocName, 0 ) ;
		
		reparse(normalizedContent, cs,comment );		
		pe.extractProperties( cs, 0 ) ;
		final ParsedCategory category = cs.getCategory();
		category.getDocumentsList().clear() ;
		reparse(normalizedContent, cs,comment);
		pe.extractProperties( cs, 0 ) ;
		
		Collection<PropertyCharacteristic> allChars = category.getChracteristicsMap().values() ;
		for( PropertyCharacteristic pChar : allChars ){
			final String propName = pChar.getName();
			final Integer index = propNamesDeviations.get( propName );
			pChar.setCharacteristic("indexInNamesArray", ""+index ) ;
		}
				
		return category ;
	}


	private String composeComment(String tableDocName, int n)
	{
		StringBuilder bld = new StringBuilder("TableInfo") ;
		bld.append(n) ;
		bld.append(":") ;
		bld.append( tableDocName ) ;
		final String result = bld.toString();
		return result ;
	}	

	private void reparse(ArrayList<ArrayList<String>> normalizedContent, final CategoryStorage cs, String comment)
	{
		IParserIdLookup pil = createparserIdLookUp(cs);
		IParserProvider dpp = DefaultParserProvider.getInstance() ;
		
		ArrayList<String> propTitles = normalizedContent.get(0) ;
		ArrayList<String> docTitles = normalizedContent.get(1) ;
		final int propsCount = propTitles.size();
		final int docsCount = docTitles.size();
		
		StringBuilder bld = new StringBuilder() ;
		
		for( int i = 0 ; i < docsCount ; i++ )
		{
			String name = docTitles.get(i) ;
			
			ParsedDocument pDoc = new ParsedDocument(name) ;
			for( int j = 0 ; j < propsCount ; j++ )
			{
				String propName = propTitles.get(j) ;
				ArrayList<String> propValues = normalizedContent.get(j+2);
				
				if( propValues.size() <= i )
					continue ;
				
				String propValue = propValues.get(i) ;
				bld.append( propName ) ;
				bld.append( " = " ) ;
				bld.append( propValue ) ;
				
				String inputString = bld.toString() ;
				bld.delete(0, bld.length() ) ;				
								
				ParsedProperty prop = new ValuesExtractorV1().processProperty(dpp, inputString, null, pil, this.storage, comment ) ;
				if( prop == null || prop.getValues().isEmpty() )
					continue ;				
					
				@SuppressWarnings("unused")
				String valId = prop.getValues().iterator().next().getId() ;
				pDoc.addProperty(prop) ;
				String newPropName = prop.getName() ;
				
				this.propNamesDeviations.put(newPropName, j) ;
			}
			cs.getCategory().addParsedDocument(pDoc) ;
		}
	}


	private IParserIdLookup createparserIdLookUp(final CategoryStorage cs) {
		return new IParserIdLookup() {
			
			@Override
			public String getParserId(String propertyName) {
				String id = cs.getParserId(propertyName) ;
				if( id == null )
					id = BasicTableInfoProcessor.this.storage.getParserId(propertyName) ;
					
				return id;
			}
		};
	}


	private OrientationInfo normalize(BasicTableInfo bti) {
		
		ArrayList<ArrayList<String>> infoContent = bti.getTablValues() ;
		
		
		OrientationInfo oi = detectOrientation( infoContent, docsMatcher ) ;
		if( oi != null )
			oi.targetName = TARGET_NAME_DOCS ;
		else{
			oi = detectOrientation( infoContent, propsMatcher ) ;
			if( oi != null )
				oi.targetName = TARGET_NAME_PROPS ;
			else
				return null ;			
		}
		
		ArrayList<ArrayList<String>> result = null ;
		
		int targetIndex = oi.targetIndex;
		if( oi.targetName == TARGET_NAME_PROPS )
		{
			ArrayList<ArrayList<String>> result_ = new ArrayList<ArrayList<String>>() ;
			if( oi.targetGoesInColumn == false )
			{			
				if( oi.transposedContent != null )
					result_ = oi.transposedContent ;
				else
					result_ = transposeContent(infoContent) ;
			}
			else		
				result_ = cloneTableInfo(infoContent) ;
			
			if( targetIndex != 0 ){
				for( ArrayList<String> lst : result_ ){
					String z_ = lst.get(0) ;
					String t_ = lst.get(targetIndex) ;
					lst.set(0, t_) ;
					lst.set(targetIndex, z_) ;					
				}				
			}
			
			for( ArrayList<String> lst : result_ ){
				ArrayList tmp ;
				if( lst.size() == 2 ){
					if(result == null)
						result = new ArrayList<ArrayList<String>>() ;					
						result.add(lst) ;
				}
				else if( lst.size() > 2){ 
					tmp = new ArrayList<String>( lst.subList(0, 2) );	
				}
				else
					return null ;
			}				
			
			oi.normalized = result ;
			return oi ;
		}
		
		if( oi.targetGoesInColumn == true )
		{			
			if( oi.transposedContent != null )
				result = oi.transposedContent ;
			else
				result = transposeContent(infoContent) ;
		}
		else		
			result = cloneTableInfo(infoContent) ;

		final int namesIndex = targetIndex ;
		if( namesIndex >= 0 )
		{
			ArrayList<String>  propNames = new ArrayList<String>() ;
			for( ArrayList<String> lst : result)
			{
				propNames.add( lst.get(0) ) ;
				lst.remove(0) ;
			}
			
			ArrayList<String> docNames = result.get(namesIndex) ;
			result.remove(namesIndex) ;
			propNames.remove(namesIndex) ;
			
			result.add(0,propNames) ;
			result.add(1, docNames) ;
			
			for( int i = 0 ; i < docNames.size() ; i++ )
				docNames.set(i, ParserV2.killWikiRefsLeaveRefs( docNames.get(i))) ;
			
			for( int i = 0 ; i < propNames.size() ; i++ )
				propNames.set(i, propNames.get(i).toLowerCase().trim() ) ;						
		}
		else{
			result.add( 0, null ) ;
			result.add( 1, null ) ;
		}
		
//		test( result, System.out ) ;
		
		int s = result.get(1).size() ; 
		for( int i = result.size(); i > 2 ; i-- )
			if( result.get(i-1).size() != s ){
				result.remove(i-1) ;
				result.get(0).remove(i-3) ;
			}
		
		oi.normalized = result ;
		return oi ;
	}
	
	private OrientationInfo detectOrientation( final ArrayList<ArrayList<String>> infoContent, Matcher matcher ) {
		
		//result >> 1 is row or column index
		//result & 1 == 1 means row
		//result & 1 == 0 means column
		OrientationInfo result = new OrientationInfo() ;
		
		ArrayList<String> rowTitles = new ArrayList<String>() ;
		for( final ArrayList<String> lst : infoContent )
			rowTitles.add( lst.get(0)) ;
		
		if( matcher.match( rowTitles ) > 50 ){
			result.targetIndex = 0 ;		
			return result ;	
		}
		
		ArrayList<String> colTitles = infoContent.get(0) ;
		if( matcher.match( colTitles ) > 50 )
		{
			result.targetGoesInColumn = false ;
			result.targetIndex = 0 ;
			return result ;
		}		
		
		ArrayList<ArrayList<String>> transposedContent = transposeContent( infoContent  );
		
		result = matcher.searchMatch( transposedContent );
		if( result != null ){			
			result.transposedContent = transposedContent ;
			return result ;
		}
		
		result = matcher.searchMatch( infoContent );
		if( result != null ){
			result.targetGoesInColumn = false ;
			return result ;
		}
		
		return null ;
	}

	private ArrayList<ArrayList<String>> transposeContent( final ArrayList<ArrayList<String>> infoContent ) {
		
		int size = infoContent.size() ;
		ArrayList<ArrayList<String>> rewrittenData = new ArrayList<ArrayList<String>>() ;
		
		for( int i = 0 ; i < infoContent.get(0).size() ; i++ )
			rewrittenData.add( new ArrayList<String>( size )) ;		
		
		final int rewrittenDataSize = rewrittenData.size();
		for( int i = 0 ; i < size ; i++ )
		{
			ArrayList<String> row = infoContent.get(i) ;
			int j = 0 ;
			for( String str : row )
			{
				if( j >= rewrittenDataSize )
					break ;
				
				rewrittenData.get(j++).add(str) ;			
			}
			for( ; j < rewrittenDataSize ; j++ )
				rewrittenData.get(j).add( null ) ;			
 		}
		return rewrittenData ;
	}

	
	
	private final static class OrientationInfo{
		
		boolean targetGoesInColumn = true ;
		int targetIndex = -1 ;
		String targetName ;
		ArrayList<ArrayList<String>> transposedContent = null ;
		ArrayList<ArrayList<String>> normalized = null ;
	}
	
	private CategoryStorage createEmptyCategoryStorage() {
		
		CategoryStorage cs = new CategoryStorage() {
			private static final long serialVersionUID = 1L;

			@Override
			protected IProperty getSpecialProperty(String propertyName) {
					return null;
			}
			
			@Override
			protected IThing createThing(Object name) {
				final String nameStr = name.toString() ;
				return new IThing() {
					
					@Override
					public String getId() {
						return nameStr ;
					}
					
					@Override
					public IRealm<IType> getTypes() {
						return null;
					}

					@Override
					public boolean isDepricated() {
						return false;
					}

					@Override
					public IProperty getProprty(String id) {
						return null;
					}

					@Override
					public Object getLongDefaultImage() {
						// TODO Auto-generated method stub
						return null;
					}
				};
			}
			
			@Override
			protected IProperty createProperty(PropertyCharacteristic pChar) {
				return null;
			}
			
			@Override
			protected void addSpecialProperties(Collection<IProperty> col) {
			}
		};
		final String path = this.storage.getCategory().getRootPath();
		cs.setCategory( new ParsedCategory( path ){
			protected Map<String,PropertyCharacteristic> createCharacteristicsMap() {
				return new LinkedHashMap<String, PropertyCharacteristic>();
			};
		} ) ;
		return cs ;
	}
	


	private ArrayList<ArrayList<String>> cloneTableInfo(
			ArrayList<ArrayList<String>> infoContent) {
		
		ArrayList<ArrayList<String>> result = new ArrayList<ArrayList<String>>() ;
		for( ArrayList<String> lst : infoContent )
			result.add( new ArrayList<String>( lst ) ) ;
		
		return result;
	}

	
	private void test(ArrayList<ArrayList<String>> result, PrintStream ps )
	{		
		final ArrayList<String> propNames = result.get(0);		
		final ArrayList<String> docNames = result.get(1);
		
		if( propNames != null && docNames != null )
		{		
			final int propsCount = propNames.size();
			final int docsCount = docNames.size();		
			
			for( int i = 0 ; i < propsCount ; i++ )
			{
				ps.print( "property: " );
				ps.println( propNames.get(i) );
				ArrayList<String> propValues = result.get(2+i) ;
				if( propValues.size() < docsCount )
					continue ;
				
				for( int j = 0 ; j < docsCount ; j++ )
				{
					ps.print( docNames.get(j) );
					ps.print( ": " );
					ps.println( propValues.get(j) );
				}
				ps.println();
			}
			ps.println("end of table") ;
			ps.println() ;
		}
		else{
			
			for( ArrayList<String> lst : result ){
				
				if( lst == null )
				{
					System.out.println( "null" ) ;
					continue ;
				}
				
				for( String str : lst )
				{
					System.out.print( str ) ;
					System.out.print( "; " ) ;
				}
				System.out.println() ;
			}
		}
	}
	
	private static class Matcher {
		
		private final HashSet<String> valSet ;
		
		private Matcher( Collection<String> inputSet ) {

			HashSet<String> vlaSet_ = new HashSet<String>() ;
			for( String s : inputSet ) {
				if( s == null )
					continue ;
				
				String canonized = canonize( s );
				if( canonized.length() == 0 )
					continue ;
				
				vlaSet_.add( canonized ) ;
			}
			valSet = vlaSet_ ;
		}
		
		private OrientationInfo searchMatch( ArrayList<ArrayList<String>> rewrittenData )
		{
			HashMap<Integer,Integer> countMap = new HashMap<Integer, Integer>() ;
			int maxWeight = -1 ;
			int namesIndex = -1 ;
			for( int i = 0 ; i < rewrittenData.size() ; i++ ){
				
				int weight = match( rewrittenData.get(i) ) ;
				if( weight < maxWeight )
					continue ;
				
				if( weight == maxWeight ){
					int count = countMap.get(maxWeight) ;
					countMap.put(maxWeight, count+1) ;
					continue ;
				}
				
				maxWeight = weight ;
				countMap.put(maxWeight,1) ;
				namesIndex = i ;
			}
			if( maxWeight < 50 )
				return null ;
			
			if( countMap.get(maxWeight) > 1 )
				return null ;
			
			if( namesIndex >= 0 )
			{
				OrientationInfo result = new OrientationInfo() ;
				result.targetIndex = namesIndex ;			
				return result ;
			}
			return null ;
		}		
		
		private int match( ArrayList<String> lst ) {
			
			int count = 0 ;
			for( String s : lst ){
				
				if( s == null )
					continue ;
				
				String str = canonize(s) ;				
							
				if( str != null )
					str = canonize( str ) ;			
			
				if( valSet.contains(str) )
					count++ ;				
			}			
			
			if( count == 0 )
				return -100 ;
			
			int percentage = (int)( 100 * ((double)count )/ lst.size() + 0.5 );
			percentage += (int)( 20 / lst.size() ) ;
			return percentage ;
		}		

		protected String canonize(String str ) {
			String s = ParserV2.killWikiRefsLeaveRefs( str.replace(" ", "_").toLowerCase() ).trim() ;	
			return s ;
		}		
	}
}
